from pathlib import Path
import re
import pdb

# 读取txt文件，将重复的行去掉，然后保存到新的txt文件中
def unique_filter(input_file: Path, output_file: Path):
    # 读取txt文件
    with open(input_file, 'r', encoding="utf-8") as f:
        lines = {line.strip() for line in f.readlines()}
    # 去掉重复的行
    lines = list(set(lines))

    # 保存到新的txt文件中
    with open(output_file, 'w', encoding="utf-8") as f:
        for line in lines:
            f.write(line + '\n')


def reformat_text(text):
    # 正则表达式匹配点号后面跟随的字母和冒号，并在后面跟着任意字符直到下一个点号或参考答案
    pattern = r'[0-9].*?(?=[A-Z]|$)|[A-Z]:.*?(?=[A-Z]|$|参考答案)|参考答案.*?[A-Z]'
    # 使用正则替换，在匹配到的字符串后面加上换行符
    matches = re.findall(pattern, text)

    return matches

# 处理数据
def process_data(input_file: Path, output_file: Path):
    # 读取txt文件
    with open(input_file, 'r', encoding="utf-8") as file:
        # 保存到新的txt文件中
        with open(output_file, 'w', encoding="utf-8") as f:
            # 逐行读取文件内容
            for line in file:
                items = reformat_text(line)
                for item in items:
                    f.write(item + '\n')
                    # 如果item包含“参考答案”，写一个空行
                    if item.find('参考答案')!= -1:
                        f.write('\n\n')

                # # 处理每一行，例如打印出来
                # print(line.strip())  # 使用strip()方法移除行尾的换行符
                # pdb.set_trace()



if __name__ == '__main__':
    # 读取txt文件的路径
    input_file = 'input/text.txt'

    # 导出去重的txt文件的路径
    output_file = 'output/text.txt'

    # 调用函数
    # unique_filter(input_file, output_file)

    process_data(input_file, output_file)
